/**
 *
 * Copyright Pact Lab of H.I.T.
 *
 * Designed and Implemented by Grid Researching Group, 
 * Pact Lab, Harbin
 * 
 * This Project is part of the national 973 Project:
 * Internet Based Virtual Computing Environment
 *
 * http://pact518.hit.edu.cn
 * 
 * Author:       Meteor <meteorlxk@gmail.com> 
 * Copyright:    pact518 
 * Version:      1.0
 * Created:      2008-12-15 
 * LastModified: 2008-12-15
 */
package edu.hit.pact.pgse.crawler.util;

import java.net.MalformedURLException;
import java.net.URL;
import java.util.ArrayList;
import java.util.HashMap;
import java.util.List;

import org.htmlparser.Node;
import org.htmlparser.NodeFilter;
import org.htmlparser.Parser;
import org.htmlparser.beans.StringBean;
import org.htmlparser.filters.NodeClassFilter;
import org.htmlparser.filters.OrFilter;
import org.htmlparser.lexer.Lexer;
import org.htmlparser.lexer.Page;
import org.htmlparser.tags.LinkTag;
import org.htmlparser.util.NodeList;
import org.htmlparser.util.ParserException;
import org.htmlparser.visitors.TextExtractingVisitor;

import edu.hit.pact.pgse.util.ChineseCharEncode;


/**
 * @author Meteor
 *
 */
public class CustonLinkBean {
	public List<URL> extractLink(URL father, String content, String charset) {
		List<URL> linkList = new ArrayList<URL>();
	
		CustomParser parser;
        parser = CustomParser.createParser(father.toString(), content, charset);
//		Parser parser = Parser.createParser(content, charset);
//		parser.setURL(father.toString());
        
        NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
        
        OrFilter lastFilter = new OrFilter();
        lastFilter.setPredicates(new NodeFilter[] { linkFilter });
        //TextExtractingVisitor visitor = new TextExtractingVisitor();
        NodeList nodelist;
        try {
        	nodelist = parser.extractAllNodesThatMatch(linkFilter);
		} catch (ParserException e1) {
			return linkList;
		}//end try
		
        Node[] nodes = nodelist.toNodeArray();
        
        String line = "";
        String word = "";
        for (Node node : nodes) {       
            if (node instanceof LinkTag) {
                LinkTag link = (LinkTag) node;
                line = link.getLink();
                word = link.getLinkText();
            } else {
            	continue;
            }
            
            if (isTrimEmpty(line)) {
                continue;            	
            }
                        
	        if (line.indexOf("#") != -1){
	            continue;
	        }

			if (line.indexOf("mailto:") != -1) {
				continue;
			}
			
			if (line.toLowerCase().indexOf("javascript") != -1) {
				continue;
			}
             
			try {
				line = ChineseCharEncode.encode(line);

				URL url = new URL(line);
				
				/**
				 * TODO: Does it need normalize?
				 */
				//url = normalize(url);
				
				linkList.add(url);
			} catch (MalformedURLException e) {
			}//end try
        }//end for (Node node : nodes)
        
		return linkList;
	}
	
	
	
	///add by HCZ 2010.11.7
	///function : return the interlinks of a html page,and get the anchor word
	public List<URL> extractLink(URL father, String content, String charset,HashMap<String,URL> linksAnchor,
			ArrayList<Anchor> anchorList) {
		List<URL> linkList = new ArrayList<URL>();
	
		CustomParser parser;
        parser = CustomParser.createParser(father.toString(), content, charset);
//		Parser parser = Parser.createParser(content, charset);
//		parser.setURL(father.toString());
        
        NodeFilter linkFilter = new NodeClassFilter(LinkTag.class);
        
        OrFilter lastFilter = new OrFilter();
        lastFilter.setPredicates(new NodeFilter[] { linkFilter });
        
        NodeList nodelist;
        try {
        	nodelist = parser.extractAllNodesThatMatch(linkFilter);
		} catch (ParserException e1) {
			return linkList;
		}//end try
		
        Node[] nodes = nodelist.toNodeArray();
        
        String line = "";
        String word = "";
        for (Node node : nodes) {       
            if (node instanceof LinkTag) {
                LinkTag link = (LinkTag) node;
                line = link.getLink();
                word = link.getLinkText();
            } else {
            	continue;
            }
            
            if (isTrimEmpty(line)) {
                continue;            	
            }
                        
	        if (line.indexOf("#") != -1){
	            continue;
	        }

			if (line.indexOf("mailto:") != -1) {
				continue;
			}
			
			if (line.toLowerCase().indexOf("javascript") != -1) {
				continue;
			}
             
			try {
				line = ChineseCharEncode.encode(line);
				//word = ChineseCharEncode.encode(word);
			
				URL url = new URL(line);
				
				/**
				 * TODO: Does it need normalize?
				 */
				//url = normalize(url);
				
				linkList.add(url);
				linksAnchor.put(ChineseCharEncode.encode(word),url);
				Anchor anchor = new Anchor();
				anchor.setUrl(url);
				anchor.setText(word);
				anchorList.add(anchor);
				
			} catch (MalformedURLException e) {
			}//end try
        }//end for (Node node : nodes)
        
		return linkList;
	}
	
	
	
	//@add by HCZ 2010.11.7
	//@return : get the text of a html page
	public String extractText(URL father,String content,String charset) throws ParserException{
		
		if (null == father) {
			return null;
		} else if (null == content) {
			return null;
		}
		Parser parser = new Parser(new Lexer (new Page (content)));
		StringBean sb = new StringBean();

		sb.setLinks(false);
		sb.setReplaceNonBreakingSpaces(true);
		sb.setCollapse(true);
		
		parser.visitAllNodesWith(sb);
		String text = sb.getStrings();
		return text;
	}
	
	
    private boolean isBlank(String astr) {
        if ((null == astr) || (astr.length() == 0)) {
            return true;
        } else {
            return false;
        }
    }
    
    private boolean isTrimEmpty(String astr) {
        if ((null == astr) || (astr.length() == 0)) {
            return true;
        }
        
        if (isBlank(astr.trim())) {
            return true;
        }
        
        return false;
    }

//    /**
//     * @param url
//     * @return
//     * @throws MalformedURLException
//     */
//    private URL normalize(URL url) throws MalformedURLException {
//    	boolean isChange = false;
//    	StringBuffer path = new StringBuffer(url.getPath());
//    	
//    	int pos;
//		while ((pos = path.indexOf("/..")) != -1) {
//			isChange = true;
//			path.delete(pos, pos + 3);
//		}
//		
//		if (isChange) {
//			URL newUrl = new URL(url.getProtocol(), url.getHost(), url.getPort(), path.toString());
//			return newUrl;
//		} else {
//			return url;
//		}
//    }   
}
